import collections
import gc
import glob

from concurrent.futures import ProcessPoolExecutor, as_completed

from absl.flags import FLAGS
from coin.pnl.sim_stat_frompb import get_stats_dict, get_sstats_dict_new


def read_sim_log(path_pattern):
  futures = []
  pb_paths = list(sorted(glob.glob(path_pattern)))
  sstats_dict = collections.defaultdict(list)

  compress = len(pb_paths) >= 2 or FLAGS.compress

  if len(pb_paths) == 1 or FLAGS.analyzer_cpu == 1:
    for pb_path in pb_paths:
      for key, sstats in get_stats_dict(pb_path, compress=compress).items():
        sstats_dict[key].extend(sstats)
  else:
    futures = {}
    with ProcessPoolExecutor(
          max_workers=min(FLAGS.analyzer_cpu, FLAGS.max_cpu)) as executor:
      pb_path_iter = iter(pb_paths)
      while True:
        for pb_path in pb_path_iter:
          future = executor.submit(get_stats_dict, pb_path, compress=compress)
          futures[future] = True
          if len(futures) >= FLAGS.analyzer_cpu:
            break
        if len(futures) == 0:
          break

        for future in as_completed(futures):
          for key, sstats in future.result().items():
            sstats_dict[key].extend(sstats)
          del futures[future]
          del future
          gc.collect()
          break

  sstats_dict = get_sstats_dict_new(sstats_dict)
  return sstats_dict
